#include "../include/PageLibPreprocessor.hpp"
#include "../include/PageLib.hpp"
#include <fstream>
#include <sstream>
#include <iostream>

using std::ofstream;
using std::make_pair;

void PageLibPreprocessor::doProcess()
{
    cutRedundantPages();
    buildInvertIndexTable();
}

bool func(WebPage& wp){
    return wp.getDocId() == -1;
}


void PageLibPreprocessor::cutRedundantPages()
{

    /*将i和i后面的依次比较，如果有网页跟i相似度较高，则将网页j删除*/
    for(size_t i = 0; i < _pageList.size()-1;++i)
    {
        for(size_t j = i+1; j< _pageList.size();++j)
        {
            if(_pageList[i]  == _pageList[j])
            {
                _pageList[j].reSetDocId(-1);
            }
        }
    }

    auto it = std::remove_if(_pageList.begin(), _pageList.end(), func);

    _pageList.erase(it, _pageList.end());

    /* 重置去重后留下来的网页的id，依次递增 */
    for(size_t idx = 0;idx < _pageList.size();++idx)
    {
        _pageList[idx].reSetDocId(idx + 1); 
    }
}


#if 1 

void PageLibPreprocessor::buildInvertIndexTable()
{
    double N = _pageList.size();

    map<string, double> DF;
    map<string, double> W;

    for(int idx = 0; idx < N; ++idx){
        map<string, int> tmp = _pageList[idx].getWordsMap();
        for(auto& word: tmp){
            ++DF[word.first];
        }
    }

    for(auto ita = _pageList.begin(); ita != _pageList.end(); ++ita)
    {
        //这里是一个WebPage
        W.clear();
        //每个WebPage的DF W都要重新算
        map<string, int> tmp = ita->getWordsMap();
        for(auto itb = tmp.begin(); itb != tmp.end(); ++itb)
        {
            //这里是WebPage中的其中一个词
            string word = itb->first;
            double tf = itb->second;
            double w = tf * (log(N / (DF[word] + 1)) / log(2));
            W[word] = w;
        }
        //跳出循环一个WebPage的所有word的w计算完毕
        double bound = 0;
        for(auto & elem : W)
        {
            bound += elem.second  * elem.second;
        }
        /* cout << "此时w平方和是 " << bound << endl; */ 

        for(auto itd = tmp.begin(); itd != tmp.end(); ++itd)
        {
            string word = itd->first;
            double w = (W[word] / sqrt(bound));//这里算的是w'
            _invertIndexTable[word].push_back(make_pair(ita->getDocId(), w));
        }
    }
}

#endif


void PageLibPreprocessor::storeOnDisk()
{
    const string ripepagePath = "../lib/ripepage.dat";
    const string offsetPath = "../lib/offset.dat";
    ofstream ofsRipe(ripepagePath, std::ios::out | std::ios::trunc);
    if(!ofsRipe)
    {
        std::cerr << "PageLibPreprocessor::storeOnDisk open " << ripepagePath << " fail!" << endl;
        return;
    }

    ofstream ofsOffset(offsetPath);
    if(!ofsOffset)
    {
        std::cerr << "PageLibPreprocessor::storeOnDisk open " << offsetPath << " fail!" << endl;
        return;
    }

    size_t begin, end;
    for(size_t idx = 0; idx != _pageList.size(); ++idx)
    {
        begin = ofsRipe.tellp();
        ofsOffset << idx + 1 << " " << begin << " ";

        ofsRipe << "<doc>" 
            << "<docid>" << idx + 1 << "</docid>"
            << "<title>" << _pageList[idx].getTitle() << "</title>"
            << "<url>" << _pageList[idx].getUrl() << "</url>" 
            << "<content>";
        // 写入ripepage.dat的同时设置偏移库offset.dat

        ofsRipe << _pageList[idx].getContent(); 
        
        ofsRipe << "</content></doc>" << endl;

        end = ofsRipe.tellp();
        ofsOffset << end - begin << endl;
    }
    ofsRipe.close();
    ofsOffset.close();

    ////////////////////
    
    //倒排索引的持久化
    ofstream ofs("../lib/invertIndex.dat", std::ios::out|std::ios::trunc);
    if(!ofs.good())
    {
        perror("PageLibPreprocessor::storeOnDisk invertIndex.dat open");
        return;
    }
    
    auto it = _invertIndexTable.begin();
    for(; it != _invertIndexTable.end(); ++it)
    {
        ofs << it->first << " ";

        auto it2 = it->second.begin();
        for(; it2 != it->second.end(); ++ it2)
        {
            ofs << it2->first << " " << it2->second << " ";
        }
        ofs << endl;
    }
    ofs.close();
}

